{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shallow Neural Network in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build a shallow neural network to classify MNIST digits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set seed for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5, 3, 6, 1, 7, 2, 8, 6, 9, 4, 0, 9,\n",
       "       1, 1, 2, 4, 3, 2, 7, 3, 8, 6, 9, 0, 5, 6, 0, 7, 6, 1, 8, 7, 9, 3, 9,\n",
       "       8, 5, 9, 3, 3, 0, 7, 4, 9, 8, 0, 9, 4, 1, 4, 4, 6, 0, 4, 5, 6, 1, 0,\n",
       "       0, 1, 7, 1, 6, 3, 0, 2, 1, 1, 7, 9, 0, 2, 6, 7, 8, 3, 9, 0, 4, 6, 7,\n",
       "       4, 6, 8, 0, 7, 8, 3], dtype=uint8)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0:99]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  84, 185, 159, 151,  60,  36,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0, 222, 254, 254, 254, 254, 241, 198,\n",
       "        198, 198, 198, 198, 198, 198, 198, 170,  52,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  67, 114,  72, 114, 163, 227, 254,\n",
       "        225, 254, 254, 254, 250, 229, 254, 254, 140,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  17,  66,\n",
       "         14,  67,  67,  67,  59,  21, 236, 254, 106,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,  83, 253, 209,  18,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,  22, 233, 255,  83,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 129, 254, 238,  44,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,  59, 249, 254,  62,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0, 133, 254, 187,   5,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   9, 205, 248,  58,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0, 126, 254, 182,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,  75, 251, 240,  57,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         19, 221, 254, 166,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   3,\n",
       "        203, 254, 219,  35,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  38,\n",
       "        254, 254,  77,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  31, 224,\n",
       "        254, 115,   1,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 133, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  61, 242, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 254,\n",
       "        219,  40,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 207,\n",
       "         18,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]], dtype=uint8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(60000, 784).astype('float32')\n",
    "X_test = X_test.reshape(10000, 784).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train /= 255\n",
    "X_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.32941177,  0.72549021,  0.62352943,\n",
       "        0.59215689,  0.23529412,  0.14117648,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.87058824,  0.99607843,  0.99607843,  0.99607843,  0.99607843,\n",
       "        0.94509804,  0.7764706 ,  0.7764706 ,  0.7764706 ,  0.7764706 ,\n",
       "        0.7764706 ,  0.7764706 ,  0.7764706 ,  0.7764706 ,  0.66666669,\n",
       "        0.20392157,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.26274511,  0.44705883,\n",
       "        0.28235295,  0.44705883,  0.63921571,  0.89019608,  0.99607843,\n",
       "        0.88235295,  0.99607843,  0.99607843,  0.99607843,  0.98039216,\n",
       "        0.89803922,  0.99607843,  0.99607843,  0.54901963,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.06666667,  0.25882354,  0.05490196,  0.26274511,\n",
       "        0.26274511,  0.26274511,  0.23137255,  0.08235294,  0.9254902 ,\n",
       "        0.99607843,  0.41568628,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.32549021,  0.99215686,  0.81960785,  0.07058824,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.08627451,  0.9137255 ,\n",
       "        1.        ,  0.32549021,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.50588238,  0.99607843,  0.93333334,  0.17254902,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.23137255,  0.97647059,\n",
       "        0.99607843,  0.24313726,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.52156866,  0.99607843,  0.73333335,  0.01960784,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.03529412,  0.80392158,\n",
       "        0.97254902,  0.22745098,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.49411765,  0.99607843,  0.71372551,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.29411766,  0.98431373,\n",
       "        0.94117647,  0.22352941,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.07450981,  0.86666667,  0.99607843,  0.65098041,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.01176471,  0.79607844,  0.99607843,\n",
       "        0.85882354,  0.13725491,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.14901961,  0.99607843,  0.99607843,  0.3019608 ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.12156863,  0.87843138,  0.99607843,\n",
       "        0.4509804 ,  0.00392157,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.52156866,  0.99607843,  0.99607843,  0.20392157,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.23921569,  0.94901961,  0.99607843,\n",
       "        0.99607843,  0.20392157,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.47450981,  0.99607843,  0.99607843,  0.85882354,  0.15686275,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.47450981,  0.99607843,\n",
       "        0.81176472,  0.07058824,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_classes = 10\n",
    "y_train = keras.utils.to_categorical(y_train, n_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(64, activation='sigmoid', input_shape=(784,)))\n",
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 64)                50240     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 50,890\n",
      "Trainable params: 50,890\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50176"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(64*784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50240"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(64*784)+64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "650"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(10*64)+10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0915 - acc: 0.0895 - val_loss: 0.0911 - val_acc: 0.0955\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0908 - acc: 0.1058 - val_loss: 0.0905 - val_acc: 0.1162\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0903 - acc: 0.1406 - val_loss: 0.0901 - val_acc: 0.1513\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0899 - acc: 0.1925 - val_loss: 0.0897 - val_acc: 0.2058\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0895 - acc: 0.2401 - val_loss: 0.0893 - val_acc: 0.2424\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0891 - acc: 0.2706 - val_loss: 0.0889 - val_acc: 0.2715\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0888 - acc: 0.2905 - val_loss: 0.0886 - val_acc: 0.2879\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0884 - acc: 0.3022 - val_loss: 0.0883 - val_acc: 0.3019\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0881 - acc: 0.3102 - val_loss: 0.0879 - val_acc: 0.3093\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0877 - acc: 0.3146 - val_loss: 0.0876 - val_acc: 0.3145\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0874 - acc: 0.3180 - val_loss: 0.0872 - val_acc: 0.3183\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0871 - acc: 0.3209 - val_loss: 0.0869 - val_acc: 0.3204\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0867 - acc: 0.3236 - val_loss: 0.0865 - val_acc: 0.3237\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0864 - acc: 0.3243 - val_loss: 0.0862 - val_acc: 0.3256\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0860 - acc: 0.3264 - val_loss: 0.0858 - val_acc: 0.3274\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0857 - acc: 0.3277 - val_loss: 0.0855 - val_acc: 0.3283\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0853 - acc: 0.3289 - val_loss: 0.0851 - val_acc: 0.3297\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0850 - acc: 0.3303 - val_loss: 0.0847 - val_acc: 0.3312\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0846 - acc: 0.3325 - val_loss: 0.0844 - val_acc: 0.3342\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0842 - acc: 0.3337 - val_loss: 0.0840 - val_acc: 0.3354\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0838 - acc: 0.3353 - val_loss: 0.0836 - val_acc: 0.3385\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0835 - acc: 0.3379 - val_loss: 0.0832 - val_acc: 0.3419\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0831 - acc: 0.3404 - val_loss: 0.0828 - val_acc: 0.3456\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0827 - acc: 0.3436 - val_loss: 0.0824 - val_acc: 0.3507\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0822 - acc: 0.3464 - val_loss: 0.0820 - val_acc: 0.3559\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0818 - acc: 0.3513 - val_loss: 0.0816 - val_acc: 0.3620\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0814 - acc: 0.3556 - val_loss: 0.0811 - val_acc: 0.3668\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0810 - acc: 0.3617 - val_loss: 0.0807 - val_acc: 0.3728\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0806 - acc: 0.3667 - val_loss: 0.0802 - val_acc: 0.3776\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0801 - acc: 0.3728 - val_loss: 0.0798 - val_acc: 0.3839\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0797 - acc: 0.3791 - val_loss: 0.0793 - val_acc: 0.3913\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0792 - acc: 0.3865 - val_loss: 0.0789 - val_acc: 0.3988\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0788 - acc: 0.3932 - val_loss: 0.0784 - val_acc: 0.4060\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0783 - acc: 0.3997 - val_loss: 0.0780 - val_acc: 0.4151\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0779 - acc: 0.4092 - val_loss: 0.0775 - val_acc: 0.4234\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0774 - acc: 0.4169 - val_loss: 0.0770 - val_acc: 0.4321\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0769 - acc: 0.4251 - val_loss: 0.0766 - val_acc: 0.4394\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0765 - acc: 0.4355 - val_loss: 0.0761 - val_acc: 0.4468\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0760 - acc: 0.4432 - val_loss: 0.0756 - val_acc: 0.4560\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0755 - acc: 0.4520 - val_loss: 0.0751 - val_acc: 0.4666\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0751 - acc: 0.4625 - val_loss: 0.0746 - val_acc: 0.4746\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0746 - acc: 0.4719 - val_loss: 0.0742 - val_acc: 0.4853\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0741 - acc: 0.4803 - val_loss: 0.0737 - val_acc: 0.4950\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0736 - acc: 0.4904 - val_loss: 0.0732 - val_acc: 0.5032\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0732 - acc: 0.4991 - val_loss: 0.0727 - val_acc: 0.5118\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0727 - acc: 0.5086 - val_loss: 0.0722 - val_acc: 0.5213\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0722 - acc: 0.5156 - val_loss: 0.0718 - val_acc: 0.5285\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0717 - acc: 0.5253 - val_loss: 0.0713 - val_acc: 0.5354\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0713 - acc: 0.5326 - val_loss: 0.0708 - val_acc: 0.5427\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0708 - acc: 0.5398 - val_loss: 0.0703 - val_acc: 0.5498\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0703 - acc: 0.5478 - val_loss: 0.0698 - val_acc: 0.5554\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0698 - acc: 0.5539 - val_loss: 0.0693 - val_acc: 0.5619\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0694 - acc: 0.5607 - val_loss: 0.0689 - val_acc: 0.5678\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0689 - acc: 0.5666 - val_loss: 0.0684 - val_acc: 0.5741\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0684 - acc: 0.5725 - val_loss: 0.0679 - val_acc: 0.5785\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0680 - acc: 0.5782 - val_loss: 0.0674 - val_acc: 0.5835\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0675 - acc: 0.5831 - val_loss: 0.0670 - val_acc: 0.5894\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0670 - acc: 0.5872 - val_loss: 0.0665 - val_acc: 0.5941\n",
      "Epoch 59/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0666 - acc: 0.5921 - val_loss: 0.0660 - val_acc: 0.5985\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0661 - acc: 0.5962 - val_loss: 0.0655 - val_acc: 0.6028\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0656 - acc: 0.6003 - val_loss: 0.0651 - val_acc: 0.6068\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0652 - acc: 0.6041 - val_loss: 0.0646 - val_acc: 0.6102\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0647 - acc: 0.6080 - val_loss: 0.0641 - val_acc: 0.6136\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0642 - acc: 0.6117 - val_loss: 0.0636 - val_acc: 0.6163\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0638 - acc: 0.6152 - val_loss: 0.0632 - val_acc: 0.6189\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0633 - acc: 0.6186 - val_loss: 0.0627 - val_acc: 0.6225\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0629 - acc: 0.6220 - val_loss: 0.0623 - val_acc: 0.6257\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0624 - acc: 0.6256 - val_loss: 0.0618 - val_acc: 0.6276\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0620 - acc: 0.6288 - val_loss: 0.0613 - val_acc: 0.6304\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0615 - acc: 0.6319 - val_loss: 0.0609 - val_acc: 0.6338\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0611 - acc: 0.6345 - val_loss: 0.0604 - val_acc: 0.6362\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0606 - acc: 0.6373 - val_loss: 0.0600 - val_acc: 0.6390\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0602 - acc: 0.6397 - val_loss: 0.0595 - val_acc: 0.6420\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0597 - acc: 0.6424 - val_loss: 0.0591 - val_acc: 0.6439\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0593 - acc: 0.6446 - val_loss: 0.0586 - val_acc: 0.6467\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0589 - acc: 0.6470 - val_loss: 0.0582 - val_acc: 0.6485\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0584 - acc: 0.6497 - val_loss: 0.0577 - val_acc: 0.6496\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0580 - acc: 0.6520 - val_loss: 0.0573 - val_acc: 0.6520\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0576 - acc: 0.6548 - val_loss: 0.0569 - val_acc: 0.6553\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0571 - acc: 0.6572 - val_loss: 0.0564 - val_acc: 0.6583\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0567 - acc: 0.6598 - val_loss: 0.0560 - val_acc: 0.6605\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0563 - acc: 0.6625 - val_loss: 0.0556 - val_acc: 0.6645\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0559 - acc: 0.6652 - val_loss: 0.0551 - val_acc: 0.6676\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0555 - acc: 0.6681 - val_loss: 0.0547 - val_acc: 0.6702\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0550 - acc: 0.6713 - val_loss: 0.0543 - val_acc: 0.6741\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0546 - acc: 0.6744 - val_loss: 0.0539 - val_acc: 0.6777\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0542 - acc: 0.6776 - val_loss: 0.0535 - val_acc: 0.6815\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0538 - acc: 0.6809 - val_loss: 0.0531 - val_acc: 0.6840\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0534 - acc: 0.6841 - val_loss: 0.0527 - val_acc: 0.6875\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0530 - acc: 0.6871 - val_loss: 0.0523 - val_acc: 0.6913\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0527 - acc: 0.6901 - val_loss: 0.0519 - val_acc: 0.6952\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0523 - acc: 0.6936 - val_loss: 0.0515 - val_acc: 0.6984\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0519 - acc: 0.6973 - val_loss: 0.0511 - val_acc: 0.7025\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0515 - acc: 0.7010 - val_loss: 0.0507 - val_acc: 0.7073\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0511 - acc: 0.7042 - val_loss: 0.0503 - val_acc: 0.7101\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0507 - acc: 0.7081 - val_loss: 0.0499 - val_acc: 0.7141\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0504 - acc: 0.7114 - val_loss: 0.0496 - val_acc: 0.7191\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0500 - acc: 0.7148 - val_loss: 0.0492 - val_acc: 0.7226\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0497 - acc: 0.7187 - val_loss: 0.0488 - val_acc: 0.7262\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0493 - acc: 0.7220 - val_loss: 0.0485 - val_acc: 0.7291\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0489 - acc: 0.7248 - val_loss: 0.0481 - val_acc: 0.7336\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0486 - acc: 0.7280 - val_loss: 0.0478 - val_acc: 0.7368\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0482 - acc: 0.7308 - val_loss: 0.0474 - val_acc: 0.7395\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0479 - acc: 0.7344 - val_loss: 0.0471 - val_acc: 0.7425\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0476 - acc: 0.7374 - val_loss: 0.0467 - val_acc: 0.7460\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0472 - acc: 0.7407 - val_loss: 0.0464 - val_acc: 0.7490\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0469 - acc: 0.7432 - val_loss: 0.0460 - val_acc: 0.7522\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0466 - acc: 0.7459 - val_loss: 0.0457 - val_acc: 0.7549\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0462 - acc: 0.7483 - val_loss: 0.0454 - val_acc: 0.7559\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0459 - acc: 0.7508 - val_loss: 0.0450 - val_acc: 0.7591\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0456 - acc: 0.7528 - val_loss: 0.0447 - val_acc: 0.7620\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0453 - acc: 0.7549 - val_loss: 0.0444 - val_acc: 0.7648\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0450 - acc: 0.7570 - val_loss: 0.0441 - val_acc: 0.7663\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0447 - acc: 0.7590 - val_loss: 0.0438 - val_acc: 0.7680\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0444 - acc: 0.7612 - val_loss: 0.0435 - val_acc: 0.7699\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0441 - acc: 0.7629 - val_loss: 0.0432 - val_acc: 0.7717\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0438 - acc: 0.7647 - val_loss: 0.0429 - val_acc: 0.7728\n",
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0435 - acc: 0.7665 - val_loss: 0.0426 - val_acc: 0.7756\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0432 - acc: 0.7682 - val_loss: 0.0423 - val_acc: 0.7779\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0429 - acc: 0.7700 - val_loss: 0.0420 - val_acc: 0.7790\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0426 - acc: 0.7712 - val_loss: 0.0417 - val_acc: 0.7806\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0423 - acc: 0.7727 - val_loss: 0.0414 - val_acc: 0.7822\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0421 - acc: 0.7746 - val_loss: 0.0411 - val_acc: 0.7849\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0418 - acc: 0.7765 - val_loss: 0.0409 - val_acc: 0.7868\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0415 - acc: 0.7784 - val_loss: 0.0406 - val_acc: 0.7881\n",
      "Epoch 126/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0412 - acc: 0.7801 - val_loss: 0.0403 - val_acc: 0.7892\n",
      "Epoch 127/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s - loss: 0.0410 - acc: 0.7818 - val_loss: 0.0401 - val_acc: 0.7912\n",
      "Epoch 128/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0407 - acc: 0.7835 - val_loss: 0.0398 - val_acc: 0.7930\n",
      "Epoch 129/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0405 - acc: 0.7852 - val_loss: 0.0395 - val_acc: 0.7954\n",
      "Epoch 130/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0402 - acc: 0.7873 - val_loss: 0.0393 - val_acc: 0.7979\n",
      "Epoch 131/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0400 - acc: 0.7888 - val_loss: 0.0390 - val_acc: 0.7992\n",
      "Epoch 132/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0397 - acc: 0.7906 - val_loss: 0.0388 - val_acc: 0.8018\n",
      "Epoch 133/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0395 - acc: 0.7923 - val_loss: 0.0385 - val_acc: 0.8032\n",
      "Epoch 134/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0392 - acc: 0.7942 - val_loss: 0.0383 - val_acc: 0.8045\n",
      "Epoch 135/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0390 - acc: 0.7955 - val_loss: 0.0380 - val_acc: 0.8060\n",
      "Epoch 136/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0387 - acc: 0.7972 - val_loss: 0.0378 - val_acc: 0.8077\n",
      "Epoch 137/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0385 - acc: 0.7989 - val_loss: 0.0376 - val_acc: 0.8090\n",
      "Epoch 138/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0383 - acc: 0.8008 - val_loss: 0.0373 - val_acc: 0.8103\n",
      "Epoch 139/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0380 - acc: 0.8025 - val_loss: 0.0371 - val_acc: 0.8116\n",
      "Epoch 140/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0378 - acc: 0.8043 - val_loss: 0.0369 - val_acc: 0.8135\n",
      "Epoch 141/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0376 - acc: 0.8063 - val_loss: 0.0366 - val_acc: 0.8143\n",
      "Epoch 142/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0374 - acc: 0.8081 - val_loss: 0.0364 - val_acc: 0.8165\n",
      "Epoch 143/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0371 - acc: 0.8095 - val_loss: 0.0362 - val_acc: 0.8186\n",
      "Epoch 144/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0369 - acc: 0.8112 - val_loss: 0.0360 - val_acc: 0.8197\n",
      "Epoch 145/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0367 - acc: 0.8127 - val_loss: 0.0358 - val_acc: 0.8216\n",
      "Epoch 146/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0365 - acc: 0.8142 - val_loss: 0.0355 - val_acc: 0.8227\n",
      "Epoch 147/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0363 - acc: 0.8158 - val_loss: 0.0353 - val_acc: 0.8243\n",
      "Epoch 148/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0361 - acc: 0.8168 - val_loss: 0.0351 - val_acc: 0.8264\n",
      "Epoch 149/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0359 - acc: 0.8185 - val_loss: 0.0349 - val_acc: 0.8275\n",
      "Epoch 150/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0357 - acc: 0.8198 - val_loss: 0.0347 - val_acc: 0.8283\n",
      "Epoch 151/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0355 - acc: 0.8214 - val_loss: 0.0345 - val_acc: 0.8302\n",
      "Epoch 152/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0353 - acc: 0.8226 - val_loss: 0.0343 - val_acc: 0.8312\n",
      "Epoch 153/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0351 - acc: 0.8235 - val_loss: 0.0341 - val_acc: 0.8327\n",
      "Epoch 154/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0349 - acc: 0.8247 - val_loss: 0.0339 - val_acc: 0.8343\n",
      "Epoch 155/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0347 - acc: 0.8260 - val_loss: 0.0337 - val_acc: 0.8359\n",
      "Epoch 156/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0345 - acc: 0.8270 - val_loss: 0.0336 - val_acc: 0.8372\n",
      "Epoch 157/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0344 - acc: 0.8285 - val_loss: 0.0334 - val_acc: 0.8390\n",
      "Epoch 158/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0342 - acc: 0.8296 - val_loss: 0.0332 - val_acc: 0.8398\n",
      "Epoch 159/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0340 - acc: 0.8306 - val_loss: 0.0330 - val_acc: 0.8410\n",
      "Epoch 160/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0338 - acc: 0.8317 - val_loss: 0.0328 - val_acc: 0.8416\n",
      "Epoch 161/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0336 - acc: 0.8327 - val_loss: 0.0326 - val_acc: 0.8420\n",
      "Epoch 162/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0335 - acc: 0.8337 - val_loss: 0.0325 - val_acc: 0.8434\n",
      "Epoch 163/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0333 - acc: 0.8347 - val_loss: 0.0323 - val_acc: 0.8443\n",
      "Epoch 164/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0331 - acc: 0.8357 - val_loss: 0.0321 - val_acc: 0.8455\n",
      "Epoch 165/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0330 - acc: 0.8368 - val_loss: 0.0320 - val_acc: 0.8462\n",
      "Epoch 166/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0328 - acc: 0.8381 - val_loss: 0.0318 - val_acc: 0.8471\n",
      "Epoch 167/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0326 - acc: 0.8387 - val_loss: 0.0316 - val_acc: 0.8486\n",
      "Epoch 168/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0325 - acc: 0.8398 - val_loss: 0.0315 - val_acc: 0.8497\n",
      "Epoch 169/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0323 - acc: 0.8406 - val_loss: 0.0313 - val_acc: 0.8506\n",
      "Epoch 170/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0321 - acc: 0.8416 - val_loss: 0.0311 - val_acc: 0.8510\n",
      "Epoch 171/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0320 - acc: 0.8424 - val_loss: 0.0310 - val_acc: 0.8516\n",
      "Epoch 172/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0318 - acc: 0.8431 - val_loss: 0.0308 - val_acc: 0.8523\n",
      "Epoch 173/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0317 - acc: 0.8436 - val_loss: 0.0307 - val_acc: 0.8530\n",
      "Epoch 174/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0315 - acc: 0.8442 - val_loss: 0.0305 - val_acc: 0.8539\n",
      "Epoch 175/200\n",
      "60000/60000 [==============================] - ETA: 0s - loss: 0.0314 - acc: 0.844 - 1s - loss: 0.0314 - acc: 0.8452 - val_loss: 0.0304 - val_acc: 0.8549\n",
      "Epoch 176/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0312 - acc: 0.8460 - val_loss: 0.0302 - val_acc: 0.8550\n",
      "Epoch 177/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0311 - acc: 0.8467 - val_loss: 0.0301 - val_acc: 0.8558\n",
      "Epoch 178/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0309 - acc: 0.8475 - val_loss: 0.0299 - val_acc: 0.8561\n",
      "Epoch 179/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0308 - acc: 0.8483 - val_loss: 0.0298 - val_acc: 0.8567\n",
      "Epoch 180/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0307 - acc: 0.8489 - val_loss: 0.0297 - val_acc: 0.8571\n",
      "Epoch 181/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0305 - acc: 0.8496 - val_loss: 0.0295 - val_acc: 0.8578\n",
      "Epoch 182/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0304 - acc: 0.8502 - val_loss: 0.0294 - val_acc: 0.8584\n",
      "Epoch 183/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0303 - acc: 0.8506 - val_loss: 0.0292 - val_acc: 0.8590\n",
      "Epoch 184/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0301 - acc: 0.8511 - val_loss: 0.0291 - val_acc: 0.8590\n",
      "Epoch 185/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0300 - acc: 0.8518 - val_loss: 0.0290 - val_acc: 0.8590\n",
      "Epoch 186/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0299 - acc: 0.8523 - val_loss: 0.0288 - val_acc: 0.8594\n",
      "Epoch 187/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0297 - acc: 0.8526 - val_loss: 0.0287 - val_acc: 0.8599\n",
      "Epoch 188/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0296 - acc: 0.8529 - val_loss: 0.0286 - val_acc: 0.8605\n",
      "Epoch 189/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0295 - acc: 0.8534 - val_loss: 0.0285 - val_acc: 0.8609\n",
      "Epoch 190/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0294 - acc: 0.8540 - val_loss: 0.0283 - val_acc: 0.8614\n",
      "Epoch 191/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0292 - acc: 0.8544 - val_loss: 0.0282 - val_acc: 0.8617\n",
      "Epoch 192/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0291 - acc: 0.8550 - val_loss: 0.0281 - val_acc: 0.8617\n",
      "Epoch 193/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0290 - acc: 0.8555 - val_loss: 0.0280 - val_acc: 0.8624\n",
      "Epoch 194/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0289 - acc: 0.8558 - val_loss: 0.0279 - val_acc: 0.8629\n",
      "Epoch 195/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0288 - acc: 0.8564 - val_loss: 0.0277 - val_acc: 0.8633\n",
      "Epoch 196/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0286 - acc: 0.8570 - val_loss: 0.0276 - val_acc: 0.8638\n",
      "Epoch 197/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0285 - acc: 0.8574 - val_loss: 0.0275 - val_acc: 0.8640\n",
      "Epoch 198/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0284 - acc: 0.8579 - val_loss: 0.0274 - val_acc: 0.8649\n",
      "Epoch 199/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0283 - acc: 0.8585 - val_loss: 0.0273 - val_acc: 0.8656\n",
      "Epoch 200/200\n",
      "60000/60000 [==============================] - 1s - loss: 0.0282 - acc: 0.8587 - val_loss: 0.0272 - val_acc: 0.8658\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f308e68be48>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, batch_size=128, epochs=200, verbose=1, validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9472/10000 [===========================>..] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.027176343995332718, 0.86580000000000001]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
